# coding=utf-8

# sudo apt install phantomjs
import pandas as pd
import random
import re
import time
from pyvirtualdisplay import Display
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
import codecs
import signal
import linecache
import urllib3

import sys, os


class Browser:

    driver = None
    display= None

    try_times = 3

    def __init__(self):
        self.restart()
        #self.proxy()
        pass

    def proxy(self):
        proxy = webdriver.Proxy()
        proxy.proxy_type = proxy.proxyType.MANUAL
        proxy.http_proxy = 'http://109.105.1.52:8080'
        proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
        self.driver.start_session(webdriver.DesiredCapabilities.PHANTOMJS)

    def restart(self):

        if self.driver is not None:
            self.driver.quit()
        if self.display is not None:
            self.display.stop()
        """
        os.system('pkill -9 chrome')
        os.system('pkill -9 Xvfb')
        os.system('pkill -9 chromedriver')
        os.system('rm -rf /tmp/.com.google.Chrome*')
        os.system('rm -rf /tmp/.org.chromium*')
        """
        # 模拟chrome，并消除浏览器图像化，以便在服务器里运行
        self.display = Display(visible=0, size=(800, 800))
        self.display.start()
        #self.driver = webdriver.Chrome()
        self.driver = webdriver.PhantomJS()
        #self.driver = webdriver.Firefox()


    # 获取商品页面内容,返回页面html
    def get(self, _Url):
        _try_times = self.try_times
        content = None
        while _try_times > 0:
            try:
                self.driver.get(_Url)

                #scroll to buttom
                #js = "document.body.scrollTop=100000"
                #self.driver.execute_script(js)
                #time.sleep(10)

                # print(self.driver.current_url)
                content = self.driver.page_source.encode('utf-8', 'ignore').decode()
                break
            except:
                _try_times -= 1
                time.sleep(random.randint(1, 3))
                self.restart()
        return content


class Request:
    from urllib import request
    import urllib.request, urllib.parse, urllib.error

    try_times = 3
    # request 代理
    user_agents = [
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
        "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
        "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
        "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
        "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
        "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
        "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
        "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52", ]

    def __init__(self):

        self.restart()
        #self.proxy()
        pass

    def proxy(self):
        urllib3.ProxyManager = urllib3.proxy_from_url('http://109.105.1.52:8080')

        #proxy_handler = urllib2.ProxyHandler({"http": 'http://109.105.1.52:8080', "https": 'http://109.105.1.52:8080'})
        #opener = urllib2.build_opener(proxy_handler)
        #urllib2.install_opener(opener)

    def restart(self):
        pass


    # 获取商品页面内容,返回页面html
    def get(self, _Url):
        _try_times = self.try_times
        content = None
        while _try_times > 0:
            try:
                # 随机选取一个agent
                index = random.randint(0, 15)
                user_agent = self.user_agents[index]
                head = {'User-Agent': user_agent}
                req = self.request.Request(_Url, headers=head)
                response = self.urllib.request.urlopen(req)
                content = response.read().decode('utf-8')
                break
            except RuntimeError:
                _try_times-=1
            except Exception as e:
                print("ERROR %s" % e)
                break
        return content


if __name__ == '__main__':
    #d = Browser()
    d = Request()
    content = d.get("http://www.baidu.com")
    print(content)